<!DOCTYPE HTML>
<html>

<head>
	<link rel="bookmark"  type="image/x-icon"  href="/img/logo_miccall.png"/>
	<link rel="shortcut icon" href="/img/logo_miccall.png">
	
			    <title>
    Ame雨
    </title>
    <meta charset="utf-8" />
    <meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no" />
    <link rel="stylesheet" href="/amelin-ting/css/mic_main.css" />
    <link rel="stylesheet" href="/amelin-ting/css/dropdownMenu.css" />
    <meta name="keywords" content="Ame雨" />
    
    	<script async src="//busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script>
	 
    <noscript>
        <link rel="stylesheet" href="/amelin-ting/css/noscript.css" />
    </noscript>
    <style type="text/css">
        body:before {
          content: ' ';
          position: fixed;
          top: 0;
          background: url('/amelin-ting/img/bg.jpg') center 0 no-repeat;
          right: 0;
          bottom: 0;
          left: 0;
          background-size: cover; 
        }
    </style>

			    
  
    <script type="text/x-mathjax-config">
      MathJax.Hub.Config({
        tex2jax: {
          inlineMath: [ ['$','$'], ["\\(","\\)"]  ],
          processEscapes: true,
          skipTags: ['script', 'noscript', 'style', 'textarea', 'pre', 'code']
        }
      });
    </script>

    <script type="text/x-mathjax-config">
      MathJax.Hub.Queue(function() {
        var all = MathJax.Hub.getAllJax(), i;
        for (i=0; i < all.length; i += 1) {
          all[i].SourceElement().parentNode.className += ' has-jax';
        }
      });
    </script>
    <script async type="text/javascript" src="//cdn.bootcss.com/mathjax/2.7.1/latest.js?config=TeX-AMS-MML_HTMLorMML"></script>
  


    <script src="/amelin-ting/js/jquery.min.js"></script>
    <script src="/amelin-ting/js/jquery.scrollex.min.js"></script>
    <script src="/amelin-ting/js/jquery.scrolly.min.js"></script>
    <script src="/amelin-ting/js/skel.min.js"></script>
    <script src="/amelin-ting/js/util.js"></script>
    <script src="/amelin-ting/js/main.js"></script>
	
<meta name="generator" content="Hexo 5.4.0"></head>
    
		
<!-- Layouts -->



<!--  代码渲染  -->
<link rel="stylesheet" href="/css/prism_coy.css" />
<link rel="stylesheet" href="/css/typo.css" />
<!-- 文章页 -->
<body class="is-loading">
    <!-- Wrapper 外包 s-->
    <div id="wrapper" class="fade-in">
        <!-- Intro 头部显示 s -->
        <!-- Intro 头部显示 e -->
        <!-- Header 头部logo start -->
        <header id="header">
    <a href="/amelin-ting/" class="logo">Art Space</a>
</header>
        <!-- Nav 导航条 start -->
        <nav id="nav" class="special" >
            <ul class="menu links" >
			<!-- Homepage  主页  --> 
			<li >
	            <a href="/amelin-ting/" rel="nofollow">主页</a>
	        </li>
			<!-- categories_name  分类   --> 
	        
	        <li class="active">
	            <a href="#s1">分类</a>
	                    <ul class="submenu">
	                        <li>
	                        <a class="category-link" href="/amelin-ting/categories/Go/">Go</a></li><li><a class="category-link" href="/amelin-ting/categories/django/">django</a></li><li><a class="category-link" href="/amelin-ting/categories/drf/">drf</a></li><li><a class="category-link" href="/amelin-ting/categories/gin/">gin</a></li><li><a class="category-link" href="/amelin-ting/categories/go/">go</a></li><li><a class="category-link" href="/amelin-ting/categories/linux/">linux</a></li><li><a class="category-link" href="/amelin-ting/categories/mysql/">mysql</a></li><li><a class="category-link" href="/amelin-ting/categories/python/">python</a></li><li><a class="category-link" href="/amelin-ting/categories/%E5%86%85%E7%BD%91%E7%A9%BF%E9%80%8F/">内网穿透</a></li><li><a class="category-link" href="/amelin-ting/categories/%E5%89%8D%E7%AB%AF/">前端</a>
	                    </ul>
	        </li>
	        
	        <!-- archives  归档   --> 
	        
	        
		        <!-- Pages 自定义   -->
		        
		        <li>
		            <a href="/about/" title="简历">
		                简历
		            </a>
		        </li>
		        
		        <li>
		            <a href="/group/" title="团队">
		                团队
		            </a>
		        </li>
		        
		        <li>
		            <a href="/gallery/" title="图库">
		                图库
		            </a>
		        </li>
		        
		        <li>
		            <a href="/tag/" title="标签">
		                标签
		            </a>
		        </li>
		        


            </ul>
            <!-- icons 图标   -->
			<ul class="icons">
                    
                    <li>
                        <a title="github" href="https://github.com/miccall" target="_blank" rel="noopener">
                            <i class="icon fa fa-github"></i>
                        </a>
                    </li>
                    
                    <li>
                        <a title="500px" href="http://500px.com" target="_blank" rel="noopener">
                            <i class="icon fa fa-500px"></i>
                        </a>
                    </li>
                    
			</ul>
</nav>

        <div id="main" >
            <div class ="post_page_title_img" style="height: 25rem;background-image: url(https://img0.baidu.com/it/u=2534927093,4145410187&amp;fm=26&amp;fmt=auto);background-position: center; background-repeat:no-repeat; background-size:cover;-moz-background-size:cover;overflow:hidden;" >
                <a href="#" style="padding: 4rem 4rem 2rem 4rem ;"><h2 >Go-kafka</h2></a>
            </div>
            <!-- Post -->
            <div class="typo" style="padding: 3rem;">
                <h2 id="一、kafka"><a href="#一、kafka" class="headerlink" title="一、kafka"></a>一、kafka</h2><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br></pre></td><td class="code"><pre><span class="line"><span class="number">1.</span> 什么是kafka</span><br><span class="line">	kafka是一个分布式数据流平台，可以运行在单台服务器上，也可以在多台服务区上部署形成集群。它提供了发布和订阅功能，使用者可以发送数据到kafka中，也可以从kafka中读取数据(一边进行后续的处理)。kafka具有高吞吐、低延迟、高容错等特点。</span><br><span class="line"></span><br><span class="line"><span class="number">2.</span> 架构介绍</span><br><span class="line"><span class="comment">/*</span></span><br><span class="line"><span class="comment">	Prouducer:即生产者。消息的产生者，是消息的入口</span></span><br><span class="line"><span class="comment"></span></span><br><span class="line"><span class="comment">	kafka cluster:kaffka集群，一台或多台服务器组成</span></span><br><span class="line"><span class="comment">		Broker:Broker是指部署了kafka实例的服务器节点，每个服务器上有一个或多个kafka的实例。我们姑且认为每个Broker对应一台服务器。每个kafka集群内的broker都有一个不重复的编号，比如broker-0、broker-1等...</span></span><br><span class="line"><span class="comment">		Topic:消息的主体，可以理解为消息的分类，kafka的数据就保存在topic。在每个broker上都可以创建多个topic。实际应用中通常是一个业务线建一个topic。</span></span><br><span class="line"><span class="comment">		Partition:Topic的分区，每个topic可以有多个分区，分区的作用是做负载，提高kafka的吞吐量。同一个tapic在不同的分区的数据是不重复的，partition的表现就是一个一个的文件夹。</span></span><br><span class="line"><span class="comment">		Replication:每一个分区都有多个副本，副本的作用是做备胎。当主分区(Leader)故障的时候会选择一个备胎(Follower)上位，成为Leader。在kafka中默认副本的数量是10个，且副本的数量不能大于Broker的数量，follower个leader就绝对是在不同的机器，同一机器对同一个分区也只可能存放同一个副本(包括自己)。</span></span><br><span class="line"><span class="comment"></span></span><br><span class="line"><span class="comment">	Consumer:消费者，即消息的消费方，是消息的出口</span></span><br><span class="line"><span class="comment">		Consumer Group: 我们可以将多个消费组组成一个消费者组，在kafka的设计中同一个分区的数据只能被消费者组中的某一个消费者消费。同一个消费组可以消费同一个topic的不同分区的数据，这也是为了提高kafka的吞吐量。</span></span><br><span class="line"><span class="comment">*/</span></span><br><span class="line"></span><br><span class="line"><span class="number">3.</span> 工作流程</span><br><span class="line"><span class="comment">/*</span></span><br><span class="line"><span class="comment">    1. 生产者从kafka集群获取分区leader消息</span></span><br><span class="line"><span class="comment">    2. 生产者将消息发送给leader</span></span><br><span class="line"><span class="comment">    3. leader将消息写入本地硬盘</span></span><br><span class="line"><span class="comment">    4. follower从leader拉取消息数据</span></span><br><span class="line"><span class="comment">    5. follower将消息写⼊本地磁盘后向leader发送ACK</span></span><br><span class="line"><span class="comment">    6. leader收到所有的follower的ACK之后向⽣产者发送ACK</span></span><br><span class="line"><span class="comment">*/</span></span><br><span class="line"></span><br><span class="line"><span class="number">4.</span> 选择partition的原则</span><br><span class="line">	在kafka中，如果某个topic有多个partition，producer又怎么知道该将数据发往哪个partition呢</span><br><span class="line">	kafka中有几个原则</span><br><span class="line">	<span class="number">1.</span> partition在写入的时候可以指定需要写入的partiton，如果有指定，则写入对应的partition</span><br><span class="line">	<span class="number">2.</span> 如果没有指定partition，但是设置了数据的key，则会根据key的值hash出一个partition</span><br><span class="line">	<span class="number">3.</span> 如果既没指定partition，又没有设置key，则会采用轮询方式，即每次取一小段时间的数据写入某个partition，下一小段的时间写入下一个partition。</span><br><span class="line"></span><br><span class="line"><span class="number">5.</span> ACK应答机制</span><br><span class="line">    producer在向kafka写⼊消息的时候，可以设置参数来确定是否确认kafka接收到数据，这个参数可设置</span><br><span class="line">    的值为 <span class="number">0</span> 、 <span class="number">1</span> 、 all 。 <span class="number">0</span>代表producer往集群发送数据不需要等到集群的返回，不确保消息发送成功。安全性最低但是效</span><br><span class="line">    率最⾼。</span><br><span class="line">    <span class="number">1</span>代表producer往集群发送数据只要leader应答就可以发送下⼀条，只确保leader发送成功。</span><br><span class="line">    all代表producer往集群发送数据需要所有的follower都完成从leader的同步才会发送下⼀条，确保</span><br><span class="line">    leader发送成功和所有的副本都完成备份。安全性最⾼，但是效率最低。</span><br><span class="line">    最后要注意的是，如果往不存在的topic写数据，kafka会⾃动创建topic，partition和replication的数量</span><br><span class="line">    默认配置都是<span class="number">1</span>。</span><br><span class="line"></span><br><span class="line"><span class="number">6.</span> Topic和数据⽇志</span><br><span class="line"><span class="comment">/*</span></span><br><span class="line"><span class="comment">    topic 是同⼀类别的消息记录（record）的集合。在Kafka中，⼀个主题通常有多个订阅者。对于每个主题，Kafka集群维护了⼀个分区数据⽇志⽂件结构如下：</span></span><br><span class="line"><span class="comment">    每个partition都是⼀个有序并且不可变的消息记录集合。当新的数据写⼊时，就被追加到partition的末尾。在每个partition中，每条消息都会被分配⼀个顺序的唯⼀标识，这个标识被称为offset，即偏移量。注意，Kafka只保证在同⼀个partition内部消息是有序的，在不同partition之间，并不能保证消息有序。</span></span><br><span class="line"><span class="comment"></span></span><br><span class="line"><span class="comment">    Kafka可以配置⼀个保留期限，⽤来标识⽇志会在Kafka集群内保留多⻓时间。Kafka集群会保留在保留期限内所有被发布的消息，不管这些消息是否被消费过。⽐如保留期限设置为两天，那么数据被发布到Kafka集群的两天以内，所有的这些数据都可以被消费。当超过两天，这些数据将会被清空，以便为后续的数据腾出空间。由于Kafka会将数据进⾏持久化存储（即写⼊到硬盘上），所以保留的数据⼤⼩可以设置为⼀个⽐较⼤的值。</span></span><br><span class="line"><span class="comment">*/</span></span><br><span class="line"><span class="number">7.</span> Partition结构</span><br><span class="line">	<span class="comment">/*</span></span><br><span class="line"><span class="comment">	    Partition在服务器上的表现形式就是⼀个⼀个的⽂件夹，每个partition的⽂件夹下⾯会有多组segment⽂件，每组segment⽂件⼜包含 .index ⽂件、 .log ⽂件、 .timeindex ⽂件三个⽂件，其中 .log ⽂件就是实际存储message的地⽅，⽽ .index 和 .timeindex ⽂件为索引⽂件，⽤于检索消息。</span></span><br><span class="line"><span class="comment"></span></span><br><span class="line"><span class="comment">    消费数据</span></span><br><span class="line"><span class="comment">        多个消费者实例可以组成⼀个消费者组，并⽤⼀个标签来标识这个消费者组。⼀个消费者组中的不同消费者实例可以运⾏在不同的进程甚⾄不同的服务器上。</span></span><br><span class="line"><span class="comment">        如果所有的消费者实例都在同⼀个消费者组中，那么消息记录会被很好的均衡的发送到每个消费者实例。</span></span><br><span class="line"><span class="comment">        如果所有的消费者实例都在不同的消费者组，那么每⼀条消息记录会被⼴播到每⼀个消费者实例。</span></span><br><span class="line"><span class="comment">	*/</span></span><br><span class="line"><span class="number">8.</span>上⾯介绍了Kafka的⼀些基本概念和原理，那么Kafka可以做什么呢？⽬前主流使⽤场景基本如下：</span><br><span class="line">    <span class="number">8.1</span> 消息队列（MQ）</span><br><span class="line">        在系统架构设计中，经常会使⽤消息队列（Message Queue）——MQ。MQ是⼀种跨进程的通信机制，⽤于上下游的消息传递，使⽤MQ可以使上下游解耦，消息发送上游只需要依赖MQ，逻辑上和物理上都不需要依赖其他下游服务。MQ的常⻅使⽤场景如流量削峰、数据驱动的任务依赖等等。在MQ领域，除了Kafka外还有传统的消息队列如ActiveMQ和RabbitMQ等。</span><br><span class="line"></span><br><span class="line">    <span class="number">8.2</span> 追踪⽹站活动</span><br><span class="line">        Kafka最出就是被设计⽤来进⾏⽹站活动（⽐如PV、UV、搜索记录等）的追踪。可以将不同的活动放⼊不同的主题，供后续的实时计算、实时监控等程序使⽤，也可以将数据导⼊到数据仓库中进⾏后续的离线处理和⽣成报表等。</span><br><span class="line"></span><br><span class="line">	<span class="number">8.3</span> Metrics</span><br><span class="line">        Kafka经常被⽤来传输监控数据。主要⽤来聚合分布式应⽤程序的统计数据，将数据集中后进⾏统⼀的分析和展示等。</span><br><span class="line"></span><br><span class="line">	<span class="number">8.4</span>⽇志聚合</span><br><span class="line">    	很多⼈使⽤Kafka作为⽇志聚合的解决⽅案。⽇志聚合通常指将不同服务器上的⽇志收集起来并放⼊⼀个⽇志中⼼，⽐如⼀台⽂件服务器或者HDFS中的⼀个⽬录，供后续进⾏分析处理。相⽐于Flume和Scribe等⽇志聚合⼯具，Kafka具有更出⾊的性能。</span><br></pre></td></tr></table></figure>



<h2 id="二、kafka安装后的配置"><a href="#二、kafka安装后的配置" class="headerlink" title="二、kafka安装后的配置"></a>二、kafka安装后的配置</h2><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">config --&gt; zookeeper.properties  --&gt;  dataDir=D:\GoProject\apache-zookeeper<span class="number">-3.7</span><span class="number">.0</span>\zklogs</span><br><span class="line"><span class="comment">// bin\windows\zookeeper-server-start.bat config\zookeeper.properties</span></span><br><span class="line"></span><br><span class="line">config --&gt; server.properties --&gt;  log.dirs=D:\GoProject\kafka_2<span class="number">.13</span><span class="number">-2.8</span><span class="number">.0</span>\kafkalogs</span><br><span class="line"><span class="comment">// bin\windows\kafka-server-start.bat config\server.properties</span></span><br></pre></td></tr></table></figure>

<h2 id="三、使用sarama往kafka发消息"><a href="#三、使用sarama往kafka发消息" class="headerlink" title="三、使用sarama往kafka发消息"></a>三、使用sarama往kafka发消息</h2><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">package</span> main</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> (</span><br><span class="line">	<span class="string">&quot;fmt&quot;</span></span><br><span class="line">	<span class="string">&quot;github.com/Shopify/sarama&quot;</span></span><br><span class="line">)</span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">func</span> <span class="title">main</span><span class="params">()</span></span> &#123;</span><br><span class="line">	<span class="comment">// 生产者配置</span></span><br><span class="line">	config := sarama.NewConfig() <span class="comment">// new一个消费者</span></span><br><span class="line">	config.Producer.RequiredAcks = sarama.WaitForAll <span class="comment">// ACK</span></span><br><span class="line">	config.Producer.Partitioner = sarama.NewRandomPartitioner <span class="comment">// 分区</span></span><br><span class="line">	config.Producer.Return.Successes = <span class="literal">true</span> <span class="comment">// 确认一下</span></span><br><span class="line"></span><br><span class="line">	<span class="comment">// 连接kafka</span></span><br><span class="line">	client, err := sarama.NewSyncProducer([]<span class="keyword">string</span>&#123;<span class="string">&quot;127.0.0.1:9092&quot;</span>&#125;, config)</span><br><span class="line">	<span class="keyword">if</span> err != <span class="literal">nil</span> &#123;</span><br><span class="line">		fmt.Println(<span class="string">&quot;producer closed, err:&quot;</span>, err)</span><br><span class="line">	&#125;</span><br><span class="line">	<span class="keyword">defer</span> client.Close()</span><br><span class="line"></span><br><span class="line">	<span class="comment">// 封装消息</span></span><br><span class="line">	msg := &amp;sarama.ProducerMessage&#123;&#125;</span><br><span class="line">	msg.Topic = <span class="string">&quot;shopping&quot;</span></span><br><span class="line">	msg.Value = sarama.StringEncoder(<span class="string">&quot;我俩的情事，是一首情诗&quot;</span>)</span><br><span class="line"></span><br><span class="line">	<span class="comment">// 发送消息</span></span><br><span class="line">	pid, offset, err := client.SendMessage(msg)</span><br><span class="line">	<span class="keyword">if</span> err != <span class="literal">nil</span> &#123;</span><br><span class="line">		fmt.Println(<span class="string">&quot;send msg faild, err:&quot;</span>, err)</span><br><span class="line">	&#125;</span><br><span class="line">	fmt.Printf(<span class="string">&quot;pid: %v, offset: %v&quot;</span>, pid, offset)</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>

<h2 id="四、tail包读取日志文件"><a href="#四、tail包读取日志文件" class="headerlink" title="四、tail包读取日志文件"></a>四、tail包读取日志文件</h2><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">package</span> main</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> (</span><br><span class="line">	<span class="string">&quot;fmt&quot;</span></span><br><span class="line">	<span class="string">&quot;time&quot;</span></span><br><span class="line">	<span class="string">&quot;github.com/hpcloud/tail&quot;</span></span><br><span class="line">)</span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">func</span> <span class="title">main</span><span class="params">()</span></span> &#123;</span><br><span class="line">	<span class="comment">// 文件名</span></span><br><span class="line">	fileName := <span class="string">`./xx.log`</span></span><br><span class="line">	<span class="comment">// 文件配置</span></span><br><span class="line">	config := tail.Config&#123;</span><br><span class="line">		ReOpen: <span class="literal">true</span>,</span><br><span class="line">		Follow: <span class="literal">true</span>,</span><br><span class="line">		Location: &amp;tail.SeekInfo&#123;Offset: <span class="number">0</span>, Whence: <span class="number">2</span>&#125;, <span class="comment">// 从哪个地方开始读数据</span></span><br><span class="line">		MustExist: <span class="literal">false</span>, <span class="comment">// </span></span><br><span class="line">		Poll: <span class="literal">true</span>,</span><br><span class="line">	&#125;</span><br><span class="line">	<span class="comment">// 打开文件开始读取数据</span></span><br><span class="line">	tails, err := tail.TailFile(fileName, config)</span><br><span class="line">	<span class="keyword">if</span> err != <span class="literal">nil</span> &#123;</span><br><span class="line">		fmt.Println(<span class="string">&quot;tails.err: &quot;</span>, err)</span><br><span class="line">	&#125; </span><br><span class="line">	<span class="comment">// 开始读取数据</span></span><br><span class="line">	<span class="keyword">var</span> (</span><br><span class="line">		msg *tail.Line</span><br><span class="line">		ok <span class="keyword">bool</span></span><br><span class="line">	)</span><br><span class="line">	<span class="keyword">for</span> &#123;</span><br><span class="line">		msg, ok = &lt;-tails.Lines</span><br><span class="line">		<span class="keyword">if</span> !ok &#123;</span><br><span class="line">			fmt.Printf(<span class="string">&quot;tail file close reopen, filename: %s\n&quot;</span>, tails.Filename)</span><br><span class="line">			time.Sleep(time.Second) <span class="comment">// 睡一秒，为了避免文件一直报错 浪费性能</span></span><br><span class="line">			<span class="keyword">continue</span></span><br><span class="line">		&#125;</span><br><span class="line">		fmt.Println(<span class="string">&quot;msg:&quot;</span>, msg.Text)</span><br><span class="line">	&#125;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>

<h2 id="五、配置文件版的logagent"><a href="#五、配置文件版的logagent" class="headerlink" title="五、配置文件版的logagent"></a>五、配置文件版的logagent</h2><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br><span class="line">97</span><br><span class="line">98</span><br><span class="line">99</span><br><span class="line">100</span><br><span class="line">101</span><br><span class="line">102</span><br><span class="line">103</span><br><span class="line">104</span><br><span class="line">105</span><br><span class="line">106</span><br><span class="line">107</span><br><span class="line">108</span><br><span class="line">109</span><br><span class="line">110</span><br><span class="line">111</span><br><span class="line">112</span><br><span class="line">113</span><br><span class="line">114</span><br><span class="line">115</span><br><span class="line">116</span><br><span class="line">117</span><br><span class="line">118</span><br><span class="line">119</span><br><span class="line">120</span><br><span class="line">121</span><br><span class="line">122</span><br><span class="line">123</span><br><span class="line">124</span><br><span class="line">125</span><br><span class="line">126</span><br><span class="line">127</span><br><span class="line">128</span><br><span class="line">129</span><br><span class="line">130</span><br><span class="line">131</span><br><span class="line">132</span><br><span class="line">133</span><br><span class="line">134</span><br><span class="line">135</span><br><span class="line">136</span><br><span class="line">137</span><br><span class="line">138</span><br><span class="line">139</span><br><span class="line">140</span><br><span class="line">141</span><br><span class="line">142</span><br><span class="line">143</span><br><span class="line">144</span><br><span class="line">145</span><br><span class="line">146</span><br><span class="line">147</span><br><span class="line">148</span><br><span class="line">149</span><br><span class="line">150</span><br><span class="line">151</span><br><span class="line">152</span><br><span class="line">153</span><br><span class="line">154</span><br><span class="line">155</span><br><span class="line">156</span><br><span class="line">157</span><br><span class="line">158</span><br><span class="line">159</span><br><span class="line">160</span><br><span class="line">161</span><br><span class="line">162</span><br><span class="line">163</span><br><span class="line">164</span><br><span class="line">165</span><br><span class="line">166</span><br><span class="line">167</span><br><span class="line">168</span><br><span class="line">169</span><br><span class="line">170</span><br><span class="line">171</span><br><span class="line">172</span><br><span class="line">173</span><br><span class="line">174</span><br><span class="line">175</span><br><span class="line">176</span><br><span class="line">177</span><br><span class="line">178</span><br><span class="line">179</span><br><span class="line">180</span><br><span class="line">181</span><br><span class="line">182</span><br><span class="line">183</span><br><span class="line">184</span><br><span class="line">185</span><br><span class="line">186</span><br><span class="line">187</span><br><span class="line">188</span><br><span class="line">189</span><br><span class="line">190</span><br><span class="line">191</span><br><span class="line">192</span><br><span class="line">193</span><br><span class="line">194</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">/*</span></span><br><span class="line"><span class="comment">首先需要下载这几个包:</span></span><br><span class="line"><span class="comment">    &quot;github.com/hpcloud/tail&quot;</span></span><br><span class="line"><span class="comment">    &quot;github.com/Shopify/sarama&quot;</span></span><br><span class="line"><span class="comment">    &quot;github.com/go-ini/ini&quot;</span></span><br><span class="line"><span class="comment">    &quot;github.com/sirupsen/logrus&quot;</span></span><br><span class="line"><span class="comment">文件:</span></span><br><span class="line"><span class="comment">	main.go</span></span><br><span class="line"><span class="comment">    config.ini</span></span><br><span class="line"><span class="comment">    kafka.go</span></span><br><span class="line"><span class="comment">    tailfile.go</span></span><br><span class="line"><span class="comment">*/</span></span><br><span class="line"></span><br><span class="line"><span class="comment">// kafka.go</span></span><br><span class="line">    <span class="keyword">package</span> kafka</span><br><span class="line"></span><br><span class="line">    <span class="keyword">import</span> (</span><br><span class="line">        <span class="string">&quot;github.com/Shopify/sarama&quot;</span></span><br><span class="line">        <span class="string">&quot;github.com/sirupsen/logrus&quot;</span></span><br><span class="line">    )</span><br><span class="line"></span><br><span class="line">    <span class="comment">// kafka相关操作</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line">    <span class="keyword">var</span> (</span><br><span class="line">        Client sarama.SyncProducer</span><br><span class="line">        MsgChan <span class="keyword">chan</span> *sarama.ProducerMessage</span><br><span class="line">    )</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">func</span> <span class="title">Init</span><span class="params">(address []<span class="keyword">string</span>, chanSize <span class="keyword">int64</span>)</span> <span class="params">(err error)</span></span> &#123;</span><br><span class="line">        <span class="comment">// 生产者配置</span></span><br><span class="line">        config := sarama.NewConfig()</span><br><span class="line">        config.Producer.RequiredAcks = sarama.WaitForAll <span class="comment">// ACK</span></span><br><span class="line">        config.Producer.Partitioner = sarama.NewRandomPartitioner <span class="comment">// 分区</span></span><br><span class="line">        config.Producer.Return.Successes = <span class="literal">true</span> <span class="comment">// 确认一下</span></span><br><span class="line"></span><br><span class="line">        <span class="comment">// 连接kafka</span></span><br><span class="line">        Client, err = sarama.NewSyncProducer([]<span class="keyword">string</span>&#123;<span class="string">&quot;127.0.0.1:9092&quot;</span>&#125;, config)</span><br><span class="line">        <span class="keyword">if</span> err != <span class="literal">nil</span> &#123;</span><br><span class="line">            logrus.Error(<span class="string">&quot;kafaka: producer closed, err:&quot;</span>, err)</span><br><span class="line">            <span class="keyword">return</span></span><br><span class="line">        &#125;</span><br><span class="line"></span><br><span class="line">        <span class="comment">// 初始化msgChan管道</span></span><br><span class="line">        MsgChan = <span class="built_in">make</span>(<span class="keyword">chan</span> *sarama.ProducerMessage, chanSize)</span><br><span class="line">        <span class="comment">// 起一个后台的goroutine从msgChan拿值</span></span><br><span class="line">        <span class="keyword">go</span> sendMsg()</span><br><span class="line">        <span class="keyword">return</span></span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">func</span> <span class="title">sendMsg</span><span class="params">()</span></span> &#123;</span><br><span class="line">        <span class="keyword">for</span> &#123;</span><br><span class="line">            <span class="keyword">select</span>&#123;</span><br><span class="line">            <span class="keyword">case</span> msg := &lt;- MsgChan:</span><br><span class="line">                pid, offset, err := Client.SendMessage(msg)</span><br><span class="line">                <span class="keyword">if</span> err != <span class="literal">nil</span> &#123;</span><br><span class="line">                    logrus.Warning(<span class="string">&quot;send msg failed, err&quot;</span>, err)</span><br><span class="line">                    <span class="keyword">return</span></span><br><span class="line">                &#125;</span><br><span class="line">                logrus.Infof(<span class="string">&quot;send msg to kafaka success pid %v offset %v&quot;</span>, pid,offset)</span><br><span class="line">            &#125;</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line"><span class="comment">// tailfile.go</span></span><br><span class="line">    <span class="keyword">package</span> tailfile</span><br><span class="line"></span><br><span class="line">    <span class="keyword">import</span> (</span><br><span class="line">        <span class="string">&quot;github.com/hpcloud/tail&quot;</span></span><br><span class="line">        <span class="string">&quot;github.com/sirupsen/logrus&quot;</span></span><br><span class="line">    )</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">    <span class="keyword">var</span> (</span><br><span class="line">        Tails *tail.Tail</span><br><span class="line">    )</span><br><span class="line">    <span class="comment">// tail相关</span></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">func</span> <span class="title">Init</span><span class="params">(filename <span class="keyword">string</span>)</span><span class="params">(err error)</span></span>&#123;</span><br><span class="line">        config := tail.Config&#123;</span><br><span class="line">            ReOpen: <span class="literal">true</span>,</span><br><span class="line">            Follow: <span class="literal">true</span>,</span><br><span class="line">            Location: &amp;tail.SeekInfo&#123;Offset: <span class="number">0</span>, Whence: <span class="number">2</span>&#125;,</span><br><span class="line">            MustExist: <span class="literal">false</span>,</span><br><span class="line">            Poll: <span class="literal">true</span>,</span><br><span class="line">        &#125;</span><br><span class="line">        <span class="comment">// 打开文件开始读取数据</span></span><br><span class="line">        Tails, err =  tail.TailFile(filename, config)</span><br><span class="line">        <span class="keyword">if</span> err != <span class="literal">nil</span> &#123;</span><br><span class="line">            logrus.Error(<span class="string">&quot;tailfile: create tailObj for path:%s failed, err:%v\n&quot;</span>, filename, err)</span><br><span class="line">            <span class="keyword">return</span></span><br><span class="line">        &#125;</span><br><span class="line">        <span class="keyword">return</span> </span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line"><span class="comment">// config.ini</span></span><br><span class="line">    [kafka]</span><br><span class="line">    address = <span class="number">127.0</span><span class="number">.0</span><span class="number">.1</span>:<span class="number">9092</span></span><br><span class="line">    chan_size = <span class="number">100000</span></span><br><span class="line">    topic = web_log</span><br><span class="line"></span><br><span class="line">    [collect]</span><br><span class="line">    logfile_path = D:\Go语言第三期\logs\s4.log</span><br><span class="line"></span><br><span class="line"><span class="comment">// main.go</span></span><br><span class="line">    <span class="keyword">package</span> main</span><br><span class="line"></span><br><span class="line">    <span class="keyword">import</span> (</span><br><span class="line">        <span class="string">&quot;fmt&quot;</span></span><br><span class="line">        <span class="string">&quot;logagent/kafka&quot;</span></span><br><span class="line">        <span class="string">&quot;logagent/tailfile&quot;</span></span><br><span class="line">        <span class="string">&quot;time&quot;</span></span><br><span class="line">        <span class="string">&quot;github.com/Shopify/sarama&quot;</span></span><br><span class="line">        <span class="string">&quot;github.com/go-ini/ini&quot;</span></span><br><span class="line">        <span class="string">&quot;github.com/sirupsen/logrus&quot;</span></span><br><span class="line">    )</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 所有的全局变量</span></span><br><span class="line">    <span class="keyword">type</span> Config <span class="keyword">struct</span> &#123;</span><br><span class="line">        KafkaConfig <span class="string">`ini:&quot;kafka&quot;`</span></span><br><span class="line">        Collect	<span class="string">`ini:&quot;collect&quot;`</span></span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">type</span> KafkaConfig <span class="keyword">struct</span> &#123;</span><br><span class="line">        Address <span class="keyword">string</span> <span class="string">`ini:&quot;address&quot;`</span></span><br><span class="line">        ChanSize <span class="keyword">int64</span> <span class="string">`ini:&quot;chan_size&quot;`</span></span><br><span class="line">        Topic <span class="keyword">string</span> <span class="string">`ini:&quot;topic&quot;`</span></span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">type</span> Collect <span class="keyword">struct</span> &#123;</span><br><span class="line">        LogFilePath <span class="keyword">string</span> <span class="string">`ini:&quot;logfile_path&quot;`</span></span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 真正的业务逻辑</span></span><br><span class="line">    <span class="function"><span class="keyword">func</span> <span class="title">run</span><span class="params">()</span> <span class="params">(err error)</span></span> &#123;</span><br><span class="line">        <span class="comment">// tailObj ---&gt; log ---&gt; Client ---&gt; kafka</span></span><br><span class="line">        <span class="keyword">for</span> &#123;</span><br><span class="line">            line, ok := &lt;-tailfile.Tails.Lines</span><br><span class="line">            <span class="keyword">if</span> !ok &#123;</span><br><span class="line">                logrus.Warn(<span class="string">&quot;tail file close reopen, filename: %s\n&quot;</span>, tailfile.Tails.Filename)</span><br><span class="line">                time.Sleep(time.Second) <span class="comment">// 睡一秒，为了避免文件一直报错 浪费性能</span></span><br><span class="line">                <span class="keyword">continue</span></span><br><span class="line">            &#125;</span><br><span class="line">            <span class="comment">// 利用通道将同步的代码变成异步</span></span><br><span class="line">            <span class="comment">// 把读出来的一行日志包装成kafka里面的msg类型，丢到通道中</span></span><br><span class="line">            msg := &amp;sarama.ProducerMessage&#123;&#125;</span><br><span class="line">            msg.Topic = <span class="string">&quot;web_log&quot;</span></span><br><span class="line">            msg.Value = sarama.StringEncoder(line.Text)</span><br><span class="line">            <span class="comment">// 丢到通道中</span></span><br><span class="line">            kafka.MsgChan &lt;- msg</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">func</span> <span class="title">main</span><span class="params">()</span></span> &#123;</span><br><span class="line">        <span class="keyword">var</span> configObj = <span class="built_in">new</span>(Config)</span><br><span class="line">        <span class="comment">// 读配置文件 利用go.ini</span></span><br><span class="line">        cfg, err := ini.Load(<span class="string">&quot;./conf/config.ini&quot;</span>)</span><br><span class="line">        <span class="keyword">if</span> err != <span class="literal">nil</span> &#123;</span><br><span class="line">            logrus.Error()</span><br><span class="line">            <span class="keyword">return</span></span><br><span class="line">        &#125;</span><br><span class="line">        kafkaAddr := cfg.Section(<span class="string">&quot;kafka&quot;</span>).Key(<span class="string">&quot;address&quot;</span>).String()</span><br><span class="line">        fmt.Println(kafkaAddr)</span><br><span class="line"></span><br><span class="line">        err = ini.MapTo(configObj, <span class="string">&quot;./conf/config.ini&quot;</span>)</span><br><span class="line">        <span class="keyword">if</span> err != <span class="literal">nil</span> &#123;</span><br><span class="line">            logrus.Error()</span><br><span class="line">            <span class="keyword">return</span></span><br><span class="line">        &#125;</span><br><span class="line">        fmt.Printf(<span class="string">&quot;%#v&quot;</span>, configObj)</span><br><span class="line"></span><br><span class="line">        <span class="comment">// 初始化连接kafka(做好准备工作)</span></span><br><span class="line">        err = kafka.Init([]<span class="keyword">string</span>&#123;configObj.KafkaConfig.Address&#125;, configObj.KafkaConfig.ChanSize)</span><br><span class="line">        <span class="keyword">if</span> err != <span class="literal">nil</span> &#123;</span><br><span class="line">            logrus.Error(<span class="string">&quot;init kafka failed, err: %v&quot;</span>, err)</span><br><span class="line">            <span class="keyword">return</span></span><br><span class="line">        &#125;</span><br><span class="line">        logrus.Info(<span class="string">&quot;init kafka success!&quot;</span>)</span><br><span class="line"></span><br><span class="line">        <span class="comment">// 根据日志的配置路径初始化tail</span></span><br><span class="line">        err = tailfile.Init(configObj.Collect.LogFilePath)</span><br><span class="line">        <span class="keyword">if</span> err != <span class="literal">nil</span> &#123;</span><br><span class="line">            logrus.Error(<span class="string">&quot;init tail failed, err: %v&quot;</span>, err)</span><br><span class="line">            <span class="keyword">return</span></span><br><span class="line">        &#125;</span><br><span class="line">        logrus.Info(<span class="string">&quot;init tail success!&quot;</span>)</span><br><span class="line"></span><br><span class="line">        <span class="comment">// 把日志通过sarama发往kafka</span></span><br><span class="line">        err = run()</span><br><span class="line">        <span class="keyword">if</span> err != <span class="literal">nil</span> &#123;</span><br><span class="line">            logrus.Error(<span class="string">&quot;run failed err:&quot;</span>, err)</span><br><span class="line">            <span class="keyword">return</span></span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br></pre></td></tr></table></figure>

<h2 id="六、配置文件版的logagent总结"><a href="#六、配置文件版的logagent总结" class="headerlink" title="六、配置文件版的logagent总结"></a>六、配置文件版的logagent总结</h2><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">/*</span></span><br><span class="line"><span class="comment">重要思路已经写了</span></span><br><span class="line"><span class="comment">其实就是最基本的kafka的流程 + sarame往kafka发消息 + tail包读取日志文件</span></span><br><span class="line"><span class="comment"></span></span><br><span class="line"><span class="comment"></span></span><br><span class="line"><span class="comment">main.go </span></span><br><span class="line"><span class="comment">	先利用go.ini这个包初始化配置文件</span></span><br><span class="line"><span class="comment">	然后连接kafka 初始化msgChan，因为可以利用管道把同步变成异步，起后台的goroutine去往kafka发msg</span></span><br><span class="line"><span class="comment">	在然后根据配置文件中的路径创建了一个对应的tailobj，它本身就带了一个通道</span></span><br><span class="line"><span class="comment">	最后从tailObj这个通道中一行一行读日志，然后包装成kafka的msg丢到msgChan管道中</span></span><br><span class="line"><span class="comment">	在mian函数中如果看不懂可以把检测错误的去掉，然后就看的清晰点</span></span><br><span class="line"><span class="comment">*/</span></span><br></pre></td></tr></table></figure>

<h2 id="七、kafka简单消费示例"><a href="#七、kafka简单消费示例" class="headerlink" title="七、kafka简单消费示例"></a>七、kafka简单消费示例</h2><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">package</span> main</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> (</span><br><span class="line">	<span class="string">&quot;fmt&quot;</span></span><br><span class="line">	<span class="string">&quot;github.com/Shopify/sarama&quot;</span></span><br><span class="line">	<span class="string">&quot;sync&quot;</span></span><br><span class="line">)</span><br><span class="line"></span><br><span class="line"><span class="comment">// 封装报错信息</span></span><br><span class="line"><span class="function"><span class="keyword">func</span> <span class="title">errs</span><span class="params">(err error, str <span class="keyword">string</span>)</span></span> &#123;</span><br><span class="line">    <span class="keyword">if</span> err != <span class="literal">nil</span> &#123;</span><br><span class="line">        log.Fatalf(<span class="string">&quot;%s failed of err: %v&quot;</span>, str, err)</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">func</span> <span class="title">main</span><span class="params">()</span></span> &#123;</span><br><span class="line">	<span class="comment">// 创建消费者</span></span><br><span class="line">	consumer, err := sarama.NewConsumer([]<span class="keyword">string</span>&#123;<span class="string">&quot;127.0.0.1:9092&quot;</span>&#125;, <span class="literal">nil</span>)</span><br><span class="line">	<span class="comment">// 报错消息</span></span><br><span class="line">	errs(err, <span class="string">&quot;consumer&quot;</span>)</span><br><span class="line">	<span class="comment">// 根据topic取到所有分区</span></span><br><span class="line">	partitionList, err := consumer.Partitions(<span class="string">&quot;web_log&quot;</span>)</span><br><span class="line">	errs(err, <span class="string">&quot;partitionList&quot;</span>)</span><br><span class="line">	<span class="comment">// 弄个WaitGroup来是程序保持运行等待消费</span></span><br><span class="line">	<span class="keyword">var</span> wg sync.WaitGroup</span><br><span class="line">	<span class="comment">// 遍历所有的分区</span></span><br><span class="line">	<span class="keyword">for</span> partition := <span class="keyword">range</span> partitionList &#123;</span><br><span class="line">		<span class="comment">// 针对每一个分区创建一个对应的消费者</span></span><br><span class="line">		parConsumer, error := consumer.ConsumePartition(<span class="string">&quot;web_log&quot;</span>, <span class="keyword">int32</span>(partition), sarama.OffsetNewest)</span><br><span class="line">		errs(error, <span class="string">&quot;parConsumer&quot;</span>)</span><br><span class="line">		<span class="keyword">defer</span> parConsumer.AsyncClose()</span><br><span class="line">		<span class="comment">// 异步从每个分区消费信息</span></span><br><span class="line">		wg.Add(<span class="number">1</span>)</span><br><span class="line">		<span class="keyword">go</span> <span class="function"><span class="keyword">func</span><span class="params">(sarama.PartitionConsumer)</span></span> &#123;</span><br><span class="line">			<span class="keyword">for</span> msg := <span class="keyword">range</span> parConsumer.Messages()&#123;</span><br><span class="line">				fmt.Printf(<span class="string">&quot;partition: %d, offset: %d, key: %v, value: %s\n&quot;</span>,</span><br><span class="line">					msg.Partition, msg.Offset, msg.Key, msg.Value)</span><br><span class="line">			&#125;</span><br><span class="line">		&#125;(parConsumer)</span><br><span class="line">	&#125;</span><br><span class="line">	wg.Wait()</span><br><span class="line">&#125;</span><br><span class="line"></span><br></pre></td></tr></table></figure>






            </div>

            <!-- Post Comments -->
            
    <!-- 使用 DISQUS_CLICK -->
<div id="disqus-comment">
    <div id="disqus_thread"></div>

<!-- add animation -->
<style>
	.disqus_click_btn {
            line-height: 30px;
            margin: 0;
            min-width: 50px;
            padding: 0 14px;
            display: inline-block;
            font-family: "Roboto", "Helvetica", "Arial", sans-serif;
            font-size: 14px;
            font-weight: 400;
            text-transform: uppercase;
            letter-spacing: 0;
            overflow: hidden;
            will-change: box-shadow;
            transition: box-shadow .2s cubic-bezier(.4, 0, 1, 1), background-color .2s cubic-bezier(.4, 0, .2, 1), color .2s cubic-bezier(.4, 0, .2, 1);
            outline: 0;
            cursor: pointer;
            text-decoration: none;
            text-align: center;
            vertical-align: middle;
            border: 0;
            background: rgba(158, 158, 158, .2);
            box-shadow: 0 2px 2px 0 rgba(0, 0, 0, .14), 0 3px 1px -2px rgba(0, 0, 0, .2), 0 1px 5px 0 rgba(0, 0, 0, .12);
            color: #fff;
            background-color: #7EC0EE;
            text-shadow: 0
        }
</style>
	
<div class="btn_click_load" id="disqus_bt"> 
    <button class="disqus_click_btn">点击查看评论</button>
</div>

<!--
<script type="text/javascript">
$('.btn_click_load').click(function() {
    /* * * CONFIGURATION VARIABLES: EDIT BEFORE PASTING INTO YOUR WEBPAGE * * */
    var disqus_shortname = 'http-miccall-tech'; // required: replace example with your forum shortname

    /* * * DON'T EDIT BELOW THIS LINE * * */
    (function() {
      var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true;
      dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js';
      (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq);
    })();

    document.getElementById('disqus_bt').style.display = "none";
});
</script>
-->
<script type="text/javascript">
    var disqus_config = function () {
        this.page.url = 'https://gitee.com/AmeX/amelin-ting.git/2020/05/23/Go------kafka/';  // Replace PAGE_URL with your page's canonical URL variable
        this.page.identifier = 'https://gitee.com/AmeX/amelin-ting.git/2020/05/23/Go------kafka/'; // Replace PAGE_IDENTIFIER with your page's unique identifier variable
    };
</script>

<script type="text/javascript">
    $('.btn_click_load').click(function() {  //click to load comments
        (function() { // DON'T EDIT BELOW THIS LINE
            var d = document;
            var s = d.createElement('script');
            s.src = '//http-miccall-tech.disqus.com/embed.js';
            s.setAttribute('data-timestamp', + new Date());
            (d.head || d.body).appendChild(s);
        })();
        $('.btn_click_load').css('display','none');
    });
</script>
</div>
<style>
    #disqus-comment{
        background-color: #eee;
        padding: 2pc;
    }
</style>


        </div>
        <!-- Copyright 版权 start -->
                <div id="copyright">
            <ul>
                <li>&copy;Powered By <a target="_blank" rel="noopener" href="https://hexo.io/zh-cn/" style="border-bottom: none;">hexo</a></li>
                <li>Design: <a target="_blank" rel="noopener" href="http://miccall.tech " style="border-bottom: none;">miccall</a></li>
            </ul>
            
                <span id="busuanzi_container_site_pv">本站总访问量<span id="busuanzi_value_site_pv"></span>次</span>
			
        </div>
    </div>
</body>



 	
</html>
